
# General
import numpy as np 
import pandas as pd
import plotly.express as px
import plotly
import plotly.graph_objs as go
import matplotlib.pyplot as plt
import chart_studio.plotly as py
import re

# Warnings OFF
import warnings
warnings.filterwarnings('ignore')

# Load
df = pd.read_csv("Agora.csv", encoding="latin1")
df.columns = [x.replace(" ", "") for x in df.columns] 
print("Data Load Complete")

# Change Dtype
df.Item = df.Item.astype(str)

"""
Pulizia Regex della colonna Destinazione e Origine. Ancora molto disordinato e ingannevole.

Spiegazione dell'espressione regolare:
- [a-zA-Z]{2} # Più di due lettere
- |[/] # Oppure, il carattere " / 
"""

for x in ["Origin","Destination"]:
    #1 Rimuovere le non-parole e privare le stringhe di spazi vuoti e tutti i caratteri in minuscolo e la prima lettera in maiuscolo.
    df[x] = df[x].str.capitalize().str.replace('[^\w\s]','')
    #2 Rimuovere le istanze di "solo". Informazioni ridondanti.
    df[x] = df[x].str.replace(r"\bonly\b", '').str.strip() # 2 
    #3 Trasformare tutte le iterazioni del termine "in tutto il mondo".
    df.loc[df[x].str.contains(r"(?i)world|\b(?i)word\b|(?i)global",na=False),x] = "Worldwide" # 3
    #4&5 Trasformare tutte le iterazioni di "united states" e "united kingdom".
    df.loc[df[x].str.contains(r"(?i)kingdom",na=False),x] = "Uk" # 4
    df.loc[df[x].str.contains(r"(?i)(\bunited states\b)|\b(?i)us\b",na=False),x] = "Usa" # 5
print("Cleaning of 'Origin' and 'Destination' Complete")
#

# rimozione del suffisso BTC
df["BTC"] = df['Price'].str.replace('BTC', '')

# Remove nan
df = df[pd.notnull(df['BTC'])]

#Ordinamento di alcune colonne che rappresentano le destinazioni e le origini che presentano lo stesso nome.
df['Destination'] = df['Destination'].replace('Wordwide', 'Worldwide')
df['Destination'] = df['Destination'].replace('Worlwide', 'Worldwide')
df['Destination'] = df['Destination'].replace('Anywhere', 'Worldwide')
df['Destination'] = df['Destination'].replace('Everywhere', 'Worldwide')
df['Destination'] = df['Destination'].replace('Usa', 'USA')
df['Destination'] = df['Destination'].replace('Uk', 'UK')
df['Destination'] = df['Destination'].replace('Europe', 'EU')
df['Destination'] = df['Destination'].replace('Europe union', 'EU')
df['Destination'] = df['Destination'].replace('European union', 'EU')
df['Destination'] = df['Destination'].replace('Eu', 'EU')

regex1= r'[a-zA-Z]{2}|[/]'
df = df[~df['BTC'].str.contains(regex1)]
# To float.
df.BTC = pd.to_numeric(df.BTC)

##

# Convert to Dollar Value with 2013/2015 Average (399)
# Scaling, and Rounding
df["Value"] = (df.BTC * 399).round(3)
df["LogValue"]= np.log(df.Value).round(3)
print("Bitcoin/Value Variable Cleaned")


df["Score"]= df.Rating.str.split('/').str[0]
df["Deals"] = "NaN"
df["Deals"] = df.Score[df.Score.str.contains(r"(deal)", na=False)]

df.Score[df.Score.str.contains(r"(deal)", na=False)] = "NaN"
df.Score = df.Score.str.replace(r'(~)','')
df.Score = df.Score.astype(float)
df.Deals = df.Deals.str.extract('(\d+)')
print("Rating and Deal Variable Cleaned and Expanded")

"""
parse delle categorie
"""
df = pd.concat([df,df.Category.str.split('/', expand=True)], axis=1)
df = df.rename(columns={0: 'cat1', 1: 'cat2'})
df = df.drop(columns=[2])
df = df.drop(columns=[3])

Data Load Complete
Cleaning of 'Origin' and 'Destination' Complete
Bitcoin/Value Variable Cleaned
Rating and Deal Variable Cleaned and Expanded


df.dtypes

Vendor              object
Category            object
Item                object
ItemDescription     object
Price               object
Origin              object
Destination         object
Rating              object
Remarks             object
BTC                float64
Value              float64
LogValue           float64
Score              float64
Deals               object
cat1                object
cat2                object
dtype: object

df


dfLifetime=pd.read_excel("Cartel2.xlsx")

#histogram of mean of lifetime of the Market between the opening and the closing
dfLifetime["MeanLifetime"]=dfLifetime["End"]-dfLifetime["Start"]
dfLifetime["MeanLifetime"]=dfLifetime["MeanLifetime"].dt.days

display(dfLifetime.loc[dfLifetime['Market']=='Agora'])

fig=px.histogram(dfLifetime, x="MeanLifetime", nbins=20, marginal="box")
fig.update_layout(xaxis_title="Lifetime of the Market in days")
fig.update_layout(yaxis_title="number of Market")
fig.update_layout(title_text="Distribution of the mean of lifetime of the Market in days")
fig.update_layout(font=dict(size=15))
fig.show()


dfLifetime["Closure"].value_counts()
#histogram of Closure
fig=px.histogram(dfLifetime, x="Closure", nbins=20)
fig.update_layout(xaxis_title="Reason of the closure")
fig.update_layout(title_text="Distribution of the reason of the closure")
fig.show()


#find Agora closed for legal reasons
display(dfLifetime.loc[dfLifetime["Market"]=="Agora",["Market","Closure"]])
s=dfLifetime.loc[dfLifetime["Market"]=="Agora",["Notes"]]
print(f"Ragiorne di chiusura di Agora:\n{s.iloc[0,0]}")

Ragiorne di chiusura di Agora:
Forums: lacbzxobeprssrfx.onion. Agora banned guns 2015-07-15 after ~20 gun busts became known; nevertheless, for most of its lifespan it allowed guns. Agora announced 2015-08-25 that it would be going offline indefinitely to deal with Tor deanonymization attacks; withdrawals were enabled and DNstats indicates the site went offline around 2015-09-06.


#group by origin
dfOrigin=df.groupby("Origin").count()
dfOrigin=dfOrigin.sort_values(by=['Vendor'], ascending=False)
dfOrigin=dfOrigin.reset_index()
dfOrigin=dfOrigin.rename(columns={"Vendor":"Count"})
dfOrigin=dfOrigin.head(20)
#histogram of the most common origins of products
fig=px.histogram(dfOrigin, y="Origin", x="Count",height=600)
fig.update_layout(yaxis_title="Origin")
fig.update_layout(xaxis_title="number of products")
fig.update_layout(title_text="Distribution of the most common origins of products")
fig.update_layout(yaxis={'categoryorder':'total ascending'})
fig.update_layout(font=dict(size=12.5))
fig.show()
#percentuale dell'origine dei prodotti

dfOrigin["Percent"]=dfOrigin["Count"]/dfOrigin["Count"].sum()*100
dfOrigin["Percent"]=dfOrigin["Percent"].round(2)
dfOrigin
#percentuale di prodotti con origine svizzera
display(dfOrigin.loc[dfOrigin["Origin"]=="Switzerland",["Origin","Percent"]])
display(dfOrigin.loc[dfOrigin["Origin"]=="Italy",["Origin","Percent"]])


#istogram of the most common origins of the products sold on Agora.

#group by destination
dfDestination=df.groupby("Destination").count()
dfDestination=dfDestination.sort_values(by=['Vendor'], ascending=False)
dfDestination=dfDestination.reset_index()
dfDestination=dfDestination.rename(columns={"Vendor":"Count"})
dfDestination=dfDestination.head(20)
dfDestination=dfDestination.drop([14])#drop raw 14 (You)

#histogram of the most common destinations of products
fig=px.histogram(dfDestination, y="Destination", x="Count",height=600)
fig.update_layout(yaxis_title="Destination")
fig.update_layout(xaxis_title="number of products")
fig.update_layout(title_text="Distribution of the most common destinations of products")
fig.update_layout(yaxis={'categoryorder':'total ascending'})
fig.show()


fig = plotly.tools.make_subplots(rows=1, cols=2, specs=[[{"type": "bar"}, {"type": "bar"}]])
fig.add_trace(go.Bar(name="Origin",x=dfOrigin["Origin"], y=dfOrigin["Count"]),row=1, col=1)

fig.add_trace(go.Bar(name="Destination",x=dfDestination["Destination"], y=dfDestination["Count"]),row=1, col=2)
fig.update_layout(xaxis={'categoryorder':'total descending'})
fig.update_layout(height=600,title_text="Distribution of the most common destinations and origins of products")
fig.update_layout(font=dict(size=15))
fig.show()

C:\Users\tagli\miniconda3\envs\datascience\lib\site-packages\plotly\tools.py:460: DeprecationWarning:

plotly.tools.make_subplots is deprecated, please use plotly.subplots.make_subplots instead


#group by category
dfCategory=df.groupby("cat1").count()
dfCategory=dfCategory.sort_values(by=['Vendor'], ascending=False)
dfCategory=dfCategory.reset_index()
dfCategory=dfCategory.rename(columns={"Vendor":"Count"})
dfCategory=dfCategory.head(20)
#percentuale di prodotti per categoria
dfCategory["Percent"]=dfCategory["Count"]/dfCategory["Count"].sum()*100
dfCategory["Percent"]=dfCategory["Percent"].round(2)
dfCategory
#replace Info with Information
dfCategory["cat1"]=dfCategory["cat1"].replace("Info","Information")

#istogramma che mostra come la maggior parte dei prodotti sono di tipo drugs
fig=px.histogram(dfCategory, y="cat1", x="Percent",text_auto="percent",height=600)
fig.update_layout(yaxis_title=" Global Category")
fig.update_layout(xaxis_title="Percent of products")
fig.update_layout(title_text="Distribution of the most common categorys of products")
fig.update_layout(yaxis={'categoryorder':'total ascending'})
fig.update_layout(font=dict(size=15))
fig.show()


Categorys =[]
def f (s):
    return s.split("/")[-1]


df["LastCategorys"]=df["Category"].apply(f)
#group by LastCategorys
dfLastCategorys=df.groupby("LastCategorys").count()
dfLastCategorys=dfLastCategorys.sort_values(by=['Vendor'], ascending=False)
dfLastCategorys=dfLastCategorys.reset_index()
dfLastCategorys=dfLastCategorys.rename(columns={"Vendor":"Count"})
dfLastCategorys=dfLastCategorys.head(10)
#histogram of the most common LastCategorys of products
fig=px.histogram(dfLastCategorys, y="LastCategorys", x="Count")
fig.update_layout(yaxis_title="LastCategorys")
fig.update_layout(xaxis_title="number of products")
fig.update_layout(title_text="Distribution of the most common LastCategorys of products")
fig.update_layout(yaxis={'categoryorder':'total ascending'})
fig.show()


df["Vendor"].value_counts()
display(df["Vendor"].value_counts().head(10))
display(df["Vendor"].value_counts().tail(10))

optiman          881
sexyhomer        860
mssource         823
profesorhouse    804
RXChemist        729
rc4me            648
fake             608
medibuds         604
Gotmilk          479
Bigdeal100       451
Name: Vendor, dtype: int64

agorasmora          1
MarloEscobar        1
SantasHardAtWork    1
SwissShroomery      1
rcfreedom           1
buymyshrooms        1
Fivestartravel      1
Digitalpossi2014    1
eBay                1
GermanGBL           1
Name: Vendor, dtype: int64


#plot che mostra i vendor con il maggior numero di inserzioni
fig=px.bar(df["Vendor"].value_counts().head(10), x=df["Vendor"].value_counts().head(10).index, y=df["Vendor"].value_counts().head(10).values,text=df["Vendor"].value_counts().head(10).values)
fig.update_layout(yaxis_title="Number of insertions")
fig.update_layout(xaxis_title="Vendor")
fig.update_layout(title_text="Vendor with the most insertions")
fig.update_layout(font=dict(size=15))
fig.show()


BigVendor=df["Vendor"].value_counts()>=400
df["isVendorBig"] = BigVendor[df["Vendor"]].values

MediumVendor=(df["Vendor"].value_counts()<400) & (df["Vendor"].value_counts()>=200)
df["isVendorMedium"] = MediumVendor[df["Vendor"]].values

SmallVendor=df["Vendor"].value_counts()<200
df["isVendorSmall"] = SmallVendor[df["Vendor"]].values


#define if an articol is from a big medium or small vendor
df["isVendorBig"] = BigVendor[df["Vendor"]].values
df["isVendorMedium"] = MediumVendor[df["Vendor"]].values
df["isVendorSmall"] = SmallVendor[df["Vendor"]].values

trace1 = go.Box(
    x=df.loc[df["isVendorSmall"]==True,"Score"],
    name = 'Small Vendor',
    marker = dict(
        color = 'rgb(12, 128, 12)',
    )
)
trace2 = go.Box(
    x=df.loc[df["isVendorMedium"]==True,"Score"],
    name = 'Medium Vendor',
    marker = dict(
        color = 'rgb(12, 128, 128)',
    )
)
trace3 = go.Box(
   x=df.loc[df["isVendorBig"]==True,"Score"],
    name = 'Big Vendor',
    marker = dict(
        color = 'rgb(12, 12, 140)',
    )
)
data = [trace1, trace2,trace3]
fig = go.Figure(data=data)
fig.update_layout(title_text="Box plot of the score for the different type of vendors")
fig.update_layout(xaxis_title="Score")
fig.update_layout(font=dict(size=18))
fig.show()


#count contare tutti i isVendorBig isVendorMedium isVendorSmall
display(df["isVendorBig"].value_counts())
display(df["isVendorMedium"].value_counts())
display(df["isVendorSmall"].value_counts())

False    100653
True       9022
Name: isVendorBig, dtype: int64

False    98610
True     11065
Name: isVendorMedium, dtype: int64

True     89588
False    20087
Name: isVendorSmall, dtype: int64


display(df.loc[(df["isVendorBig"]==True) & (df["Score"]==0),["Vendor"]].count())
#vedere se RXChemist a quale categoria appartiene
display(df.loc[df["Vendor"]=="RXChemist",["Vendor","isVendorBig"]])

df.loc[df["Vendor"]=="RXChemist",["Vendor","Score"]].describe()
#grafico dell'andamento dei prezzi per il vendor RXChemist
fig=px.line(df.loc[df["Vendor"]=="RXChemist",["Vendor","Score","Item"]], y="Score", x="Item", height=1000)
fig.update_layout(title_text="Price trend for the vendor RXChemist")
fig.update_layout(font=dict(size=18))
fig.show()

Vendor    21
dtype: int64


#mean of the score for each vendor

df2=df.groupby("Vendor")["Score"].mean()
#remove nan
df2=df2.dropna()

fig=px.histogram(df2, x="Score",range_x=[0,5],nbins=100)
fig.update_layout(title_text="Distribution of the score of the vendors")
fig.update_layout(font=dict(size=18))
fig.show()


from wordcloud import WordCloud, STOPWORDS

comment_words = ''
stopwords = set(STOPWORDS)
 
# iterate through the csv file
for val in df["Item"]:
     
    # typecaste each val to string
    val = str(val)
 
    # split the value
    tokens = val.split()

    for i in range(len(tokens)):
        for j in range(len(tokens[i])):
            if tokens[i][j]=="â":
                pass
    # Converts each token into lowercase
    for i in range(len(tokens)):
        if tokens[i]!= 'â' or tokens[i]!= "â":
            tokens[i] = tokens[i].lower()
     
    comment_words += " ".join(tokens)+" "
 
wordcloud = WordCloud(width = 800, height = 800,
                background_color ='white',
                stopwords = stopwords,
                min_font_size = 10).generate(comment_words)
 
# plot the WordCloud image                      
plt.figure(figsize = (8, 8), facecolor = None)
plt.imshow(wordcloud)
plt.axis("off")
plt.tight_layout(pad = 0)
 
plt.show()

	Vendor	Category	Item	ItemDescription	Price	Origin	Destination	Rating	Remarks	BTC	Value	LogValue	Score	Deals	cat1	cat2
0	CheapPayTV	Services/Hacking	12 Month HuluPlus gift Code	12-Month HuluPlus Codes for $25. They are wort...	0.05027025666666667 BTC	Torland	NaN	4.96/5	NaN	0.050270	20.058	2.999	4.96	NaN	Services	Hacking
1	CheapPayTV	Services/Hacking	Pay TV Sky UK Sky Germany HD TV and much mor...	Hi we offer a World Wide CCcam Service for En...	0.152419585 BTC	Torland	NaN	4.96/5	NaN	0.152420	60.815	4.108	4.96	NaN	Services	Hacking
2	KryptykOG	Services/Hacking	OFFICIAL Account Creator Extreme 4.2	Tagged Submission Fix Bebo Submission Fix Adju...	0.007000000000000005 BTC	Torland	NaN	4.93/5	NaN	0.007000	2.793	1.027	4.93	NaN	Services	Hacking
3	cyberzen	Services/Hacking	VPN > TOR > SOCK TUTORIAL	How to setup a VPN > TOR > SOCK super safe enc...	0.019016783532494728 BTC	NaN	NaN	4.89/5	NaN	0.019017	7.588	2.027	4.89	NaN	Services	Hacking
4	businessdude	Services/Hacking	Facebook hacking guide	. This guide will teach you how to hack Faceb...	0.062018073963963936 BTC	Torland	NaN	4.88/5	NaN	0.062018	24.745	3.209	4.88	NaN	Services	Hacking
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
109684	gonz324	Drugs/Opioids/Opium	1 gr purified Opium	This Listing is for a gramm of redefined Opium...	0.14363729 BTC	Germany	NaN	4.91/5	NaN	0.143637	57.311	4.048	4.91	NaN	Drugs	Opioids
109685	cheqdropz	Weapons/Fireworks	Shipping Ticket	in order for me to ship one of the guns you bo...	0.08680555 BTC	Usa	NaN	[0 deals]	NaN	0.086806	34.635	3.545	NaN	0	Weapons	Fireworks
109686	SnowQueen	Drugs/Opioids/Opium	0.50 GRAMS #4 White Afghani Heroin - FULL ESCROW	0.50 grams #4 White Afghani Heroin SnowQueen...	0.33641201 BTC	Canada	Worldwide	[0 deals]	NaN	0.336412	134.228	4.900	NaN	0	Drugs	Opioids
109687	SnowQueen	Drugs/Opioids/Opium	1.0 GRAMS #4 White Afghani Heroin - FULL ESCROW	1.0 grams #4 White Afghani Heroin SnowQueen ...	0.61165820 BTC	Canada	Worldwide	[0 deals]	NaN	0.611658	244.052	5.497	NaN	0	Drugs	Opioids
109688	daydreamer	Drugs/Opioids/Opium	HEROIN STAMP BAG (10pcs) BUNDLE	HEROIN STAMP BAG (10pcs) BUNDLE BEST AVAIL...	0.25491129 BTC	Usa	NaN	4.87/5	NaN	0.254911	101.710	4.622	4.87	NaN	Drugs	Opioids

Dataset Agora Marketplace¶

Introduzione¶

Dataset Principale:¶

Dataset Secondario:¶

Premesse indispensabili:¶

Set-up generale¶

Distribuzione della durata di vita dei Market¶

Ragioni per la chiusura dei negozzi online:¶

Agora motivo di chiusura:¶

Agora market, Origine più comune dei prodotti:¶

Destinazione più comine dei prodotti¶

Categorie di prodotti più comuni¶

Categorie specifiche di prodotti più comuni:¶

Analisi sui Venditori:¶

I Venditori con più inserzioni sono davvero i migliori?¶

Venditori e valutazioni per differenti categorie¶

Distribuzione delle valutazioni generali per i venditori¶

Parole più comuni nei titoli dei prodotti¶

	Vendor	isVendorBig
1375	RXChemist	True
3134	RXChemist	True
3142	RXChemist	True
3143	RXChemist	True
3146	RXChemist	True
...	...	...
102944	RXChemist	True
102945	RXChemist	True
102949	RXChemist	True
102958	RXChemist	True
109281	RXChemist	True